import csv
import os
import io
import pickle
import time


from dataclasses import dataclass, field
from collections import Counter, OrderedDict

import src.codeql_tools.viewer.codeql_variable_obj as da
import src.codeql_tools.config as config
import src.codeql_tools.const as const



#table data
VarTable={}
FuncTable={}

FuncArgsTable={}
VarStoreTable=set()
VarRefTable=set()

#view data
FuncViewByOwnerId={}
VarAssignView={}
VarPeekView={}
VarAssignHistoryView={}
VarRefView={}
FunctionOwnVarView={} 
VarRefHistoryView={}

FunctionOwnVarStore={}
FunctionOwnVarRef={}

VarIndexRefView={}

#addon data
VarIndexAddonTable=set()

@dataclass
class VariableStoreForm(da.Namespace):
    var_global_id:str=""
    var_text:str=""
    statement_type:str=""
    expression_type:str=""
    depend_var_global_id:str=""
    depend_var_text:str=""
    location:int=0
    column:int=0
    depend_var_attribute:list=field(default_factory=list)
    var_start_line:int=0
    var_start_column:int=0
    var_end_line:int=0
    var_end_column:int=0

    def is_substmt(self):
        return self.scope_names[-1] in config.no_effect_scope_name

    @property
    def global_id(self):
        name=f'{self.location}.{self.column}.{self.var_text}.{self.depend_var_text}'    
        return da.Variable.construct_global_id(self.absolute_path,self.scope_names,name)

    def __hash__(self):
        return hash(self.global_id)

    @property
    def owner_id(self):
        return da.Variable.construct_owner_id(self.absolute_path,self.scope_names)

    @property
    def token_id(self):
        return '{}.{}.{}'.format(self.depend_var_text,self.location,self.column)

    @property
    def var_token_id(self):
        return f'{self.var_text}.{self.var_end_line}.{self.var_end_column}'

    @property
    def row_id(self):
        return da.Variable.construct_row_id(self.absolute_path,self.location)

    def depend_variable_field(self):
        buffer=io.StringIO()
        buffer.write(self.depend_var_global_id)
        for e in self.depend_var_attribute:
            buffer.write("."+e)
        
        return buffer.getvalue()

@dataclass
class VariableRefForm(da.Namespace):
    var_global_id:str=""
    var_text:str=""
    statement_type:str=""
    expression_type:str=""
    location:int=0
    column:int=0
    var_attribute:list=field(default_factory=list)
    
    @property
    def global_id(self):     
        name=f'{self.location}.{self.column}.{self.var_text}'
        return da.Variable.construct_global_id(self.absolute_path,self.scope_names,name)

    @property
    def token_id(self):
        return '{}.{}.{}'.format(self.var_text,self.location,self.column)

    @property
    def owner_id(self):
        return da.Variable.construct_owner_id(self.absolute_path,self.scope_names)

    def __hash__(self):
        return hash(self.global_id)

    @property
    def row_id(self):
        return da.Variable.construct_row_id(self.absolute_path,self.location)

    def depend_variable_field(self):
        buffer=io.StringIO()
        buffer.write(self.depend_var_global_id)
        for e in self.depend_var_attribute:
            buffer.write("."+e)
        
        return buffer.getvalue()

@dataclass
class VariableAddon(da.Namespace):
    var_global_id:str=""
    var_text:str=""
    addon_global_id:str=""
    addon_text:str=""
    
    expression_type:str=""

    start_row:int=0
    start_column:int=0
    end_column:int=0
    end_row:int=0
    
    var_start_row:int=0
    var_start_column:int=0
    var_end_column:int=0
    var_end_row:int=0

    @property
    def global_id(self):
        name=f'{self.end_row}.{self.end_column}.{self.var_text}.{self.addon_text}'
        
        return da.Variable.construct_global_id(self.absolute_path,self.scope_names,name)            
    
    def __hash__(self):
        return hash(self.global_id)
    
    @property
    def owner_id(self):
        return da.Variable.construct_owner_id(self.absolute_path,self.scope_names)
        
    @property
    def token_id(self):
        return '{}.{}.{}.{}'.format(self.var_text,self.addon_text,self.end_row,self.end_column)
    
    @property
    def row_id(self):
        return da.Variable.construct_row_id(self.absolute_path,self.end_row)
    
    
@dataclass
class VariableIndex(VariableAddon):
    def __hash__(self):
        return hash(self.global_id)

@dataclass
class VariableAttr(VariableAddon):
    def __hash__(self):
        return hash(self.global_id)
    
def read_variable_store_from_csv(path):
    def safe_extract_args(target,indexes):
        for index in indexes:
            if index>=len(target):
                yield target[-1]
            else:
                yield target[index]
    
    def extract_scope(target):
        scopes=target.split(',')
        scopes=[e.split() for e in scopes]
        
        scope_types=[da.ScopeType.getScopeType(list(safe_extract_args(e,[0]))[0]) for e in scopes]
        scope_names=[list(safe_extract_args(e,[1]))[0] for e in scopes]
        
        return scope_types,scope_names
    
    def identify_var_name(row):
        args=row.split('.')
        if len(args)>1:
            var_name=args[0]
            attr=args[1:]   
            
            return var_name,attr
        else:
            return args[0],[]         
    
    result=[]
    drop_counter=0
    all_counter=0
    with open(path,'r') as f:
        reader=csv.DictReader(f)
        columns=reader.fieldnames

        for row in reader:
            all_counter+=1
            absolute_path=da.get_absolute_path_mapping(row[columns[0]])
            module_name,=safe_extract_args(row[columns[1]].split(),[1])
            scope_types,scope_names=extract_scope(row[columns[7]])
            varName=row[columns[3]]
            stmt_type=row[columns[4]]
            expression_type=row[columns[5]]
            dependVarName,attr=identify_var_name(row[columns[6]])
            depend_var_text=row[columns[6]]
            row_id,column_id=safe_extract_args(row[columns[8]].split(':'),[-1,-2])
            var_start_line,var_start_column,var_end_column,var_end_row=safe_extract_args(row[columns[9]].split(':'),[0,1,2,3])
            
            
            var_id=get_variable_id(absolute_path,scope_names,varName)
            depend_var_id=get_variable_id(absolute_path,scope_names,dependVarName)
            

            if var_id == "" or var_id is None:
                drop_counter+=1
            
            if depend_var_text in const.empty_str:
                depend_var_text=""
                expression_type="" 
                
            if varName in const.empty_str:
                varName=""   
            
            
            var=VariableStoreForm(
                absolute_path,
                scope_names,
                scope_types,
                var_id if var_id else "",
                varName,  
                stmt_type,
                expression_type,
                depend_var_id if depend_var_id else "",
                depend_var_text,
                int(row_id),
                int(column_id),
                attr,
                int(var_start_line),
                int(var_start_column),
                int(var_end_column),
                int(var_end_row))
            result.append(var)   
        
        print("detect variable store sentence {}.".format(all_counter))         
        print("legal variable store sentence {}".format(len(result)))
    return result

def read_variable_ref_from_csv(path):
    def safe_extract_args(target,indexes):
        for index in indexes:
            if index>=len(target):
                yield target[-1]
            else:
                yield target[index]
    
    def extract_scope(target):
        scopes=target.split(',')
        scopes=[e.split() for e in scopes]
        
        scope_types=[da.ScopeType.getScopeType(list(safe_extract_args(e,[0]))[0]) for e in scopes]
        scope_names=[list(safe_extract_args(e,[1]))[0] for e in scopes]
        
        return scope_types,scope_names
    
    def identify_var_name(row):
        args=row.split('.')
        if len(args)>1:
            var_name=args[0]
            attr=args[1:]   
            
            return var_name,attr
        else:
            return args[0],[]         
    
    result=[]
    drop_counter=0
    all_counter=0
    with open(path,'r') as f:
        reader=csv.DictReader(f)
        columns=reader.fieldnames

        for row in reader:
            all_counter+=1
            absolute_path=da.get_absolute_path_mapping(row[columns[0]])
            module_name,=safe_extract_args(row[columns[1]].split(),[1])
            scope_types,scope_names=extract_scope(row[columns[6]])
            varName,attr=identify_var_name(row[columns[3]])
            stmt_type=row[columns[4]]
            expression_type=row[columns[5]]
            location,column=safe_extract_args(row[columns[7]].split(':'),[-1,-2])
            
            var_id=get_variable_id(absolute_path,scope_names,varName)
            

            if var_id == "" or var_id is None:
                drop_counter+=1
                #continue
                
            if varName in const.empty_str:
                continue
            
            
            var=VariableRefForm(
                absolute_path,
                scope_names,
                scope_types,
                var_id if var_id else "",
                varName,
                stmt_type,
                expression_type,
                int(location),
                int(column),
                attr
                )
            result.append(var)   
        
        print("detect variable ref sentence {}.".format(all_counter))         
        print("legal variable ref sentence {}".format(len(result)))
    return result

def read_variable_index_from_csv(path):
    def safe_extract_args(target,indexes):
        for index in indexes:
            yield target[index%len(target)]
    
    def extract_scope(target):
        scopes=target.split(',')
        scopes=[e.split() for e in scopes]
        
        scope_types=[da.ScopeType.getScopeType(list(safe_extract_args(e,[0]))[0]) for e in scopes]
        scope_names=[list(safe_extract_args(e,[1]))[0] for e in scopes]
        
        return scope_types,scope_names
    
    def identify_var_name(row):
        args=row.split('.')
        if len(args)>1:
            var_name=args[0]
            attr=args[1:]   
            
            return var_name,attr
        else:
            return args[0],[]         
    
    result=[]
    warn_counter=0
    all_counter=0
    with open(path,'r') as f:
        reader=csv.DictReader(f)
        columns=reader.fieldnames

        for row in reader:
            all_counter+=1
            absolute_path=da.get_absolute_path_mapping(row[columns[0]])
            module_name,=safe_extract_args(row[columns[1]].split(),[1])
            scope_types,scope_names=extract_scope(row[columns[6]])
            objName,_=identify_var_name(row[columns[3]])
            indexName,_=identify_var_name(row[columns[4]])
            expression_type=row[columns[5]]
            obj_start_line,obj_start_column,obj_end_column,obj_end_row=safe_extract_args(row[columns[7]].split(':'),[0,1,2,3])
            index_start_line,index_start_column,index_end_column,index_end_row=safe_extract_args(row[columns[8]].split(':'),[0,1,2,3])
            
            obj_id=get_variable_id(absolute_path,scope_names,objName)
            index_id=get_variable_id(absolute_path,scope_names,indexName)

            if obj_id == "" or obj_id is None or index_id=="" or index_id is None:
                warn_counter+=1
                #continue
                
            if objName in const.empty_str:
                continue
            
            
            var=VariableIndex(
                absolute_path,
                scope_names,
                scope_types,
                obj_id if obj_id else "",
                objName,
                index_id if index_id else "",
                indexName,          
                expression_type,
                int(index_start_line),
                int(index_start_column),
                int(index_end_column),
                int(index_end_row),
                int(obj_start_line),
                int(obj_start_column),
                int(obj_end_column),
                int(obj_end_row)               
                )
            result.append(var)   
        
        print("detect variable index {}.".format(all_counter))         
        print("legal variable index {}".format(len(result)))
    return result

def get_variable_id(modName,scopeName,dependVarName):
    possible_var_id=da.Variable.construct_global_id(modName,scopeName,dependVarName)
    if get_var(possible_var_id):
        return possible_var_id
    else:
        return None

def add_variable_file(path):
    global VarTable
    var_seq=da.read_variable_from_csv(path)
    da.build_variable_table(var_seq,VarTable)

def add_function_file(path):
    global VarTable,FuncTable
    func_seq=da.read_function_from_csv(path)
    da.build_function_table(func_seq,FuncTable)

def add_variable_store_file(path):
    global VarStoreTable
    store_seq=read_variable_store_from_csv(path)
    
    for var_store in store_seq:
        VarStoreTable.add(var_store)
             
    print("now has {} variable store sentence".format(len(VarStoreTable)))

def add_function_args_file(path):
    global FuncArgsTable
    function_args_seq=da.read_function_args_from_csv(path)
    da.build_function_args_table(function_args_seq,FuncArgsTable)

def add_variable_ref_file(path):
    global VarRefTable
    store_seq=read_variable_ref_from_csv(path)
    
    for e in store_seq:
        VarRefTable.add(e)
             
    print("now has {} variable ref sentence".format(len(VarRefTable)))

def add_variable_index_file(path):
    global VarIndexAddonTable
    store_seq=read_variable_index_from_csv(path)
    
    for e in store_seq:
        VarIndexAddonTable.add(e)

def select(prefix):
    result={}
    for key in VarTable:
        if key.startsWith(prefix):
            result[key]=VarTable[key]
    
    return result

def build_variable_viewer():
    # def list_depend_var_by_location(var_id):
    #     if var_id not in VarAssignView:
    #         return {}
        
    #     variable_store_form=VarAssignView[var_id]
    #     variable_store_form=sorted(variable_store_form,key=lambda s:s.location,reverse=False)
        
    #     result=OrderedDict()
    #     for e in variable_store_form:
    #         if e.location not in result:
    #             result[e.location]=[]
    #         result[e.location].append(e)
        
    #     return result
    
    # def list_var_ref_by_location(var_id):
    #     if var_id not in VarRefView:
    #         return {}
        
    #     variable_ref_forms=VarRefView[var_id]
    #     variable_ref_forms=sorted(variable_ref_forms,key=lambda s:s.location,reverse=False)
        
        
    #     result=OrderedDict()
    #     for e in variable_ref_forms:
    #         if e.location not in result:
    #             result[e.location]=[]
    #         result[e.location].append(e)
            
    #     return result
    
    for key in FuncTable:
        func_obj=FuncTable[key]
        FuncViewByOwnerId[func_obj.get_function_owner_id()]=func_obj
    
    for var_store_obj in VarStoreTable:
        # var_id=var_store_obj.var_global_id
        # depend_var_id=var_store_obj.depend_var_global_id
        owner_id=var_store_obj.owner_id
        
        
        # if var_id !="" or var_id is not None:
        #     if var_id not in VarAssignView:
        #         VarAssignView[var_id]=[]
        #     VarAssignView[var_id].append(var_store_obj)    
        
        # if depend_var_id !="" or depend_var_id is not None:
        #     pass
            #if depend_var_id not in VarPeekView:
            #    VarPeekView[depend_var_id]=[]
                
            #VarPeekView[depend_var_id].append(var_store_obj)
        
        if owner_id not in FunctionOwnVarStore:
            FunctionOwnVarStore[owner_id]=[]
            
        FunctionOwnVarStore[owner_id].append(var_store_obj)
        
    for obj in VarRefTable:
        #var_id=obj.var_global_id
        owner_id=obj.owner_id
        # if var_id not in VarRefView:
        #     VarRefView[var_id]=[]
        
        # VarRefView[var_id].append(obj)
        
        if owner_id not in FunctionOwnVarRef:
            FunctionOwnVarRef[owner_id]=[]
        
        FunctionOwnVarRef[owner_id].append(obj)
        
    for var_id in VarTable:
        var=VarTable[var_id]
        owner_id=var.owner_id
        
        
        if owner_id not in FunctionOwnVarView:
            FunctionOwnVarView[owner_id]=[]         
        FunctionOwnVarView[owner_id].append(var)     
    
    for obj in VarIndexAddonTable:
        
        row_id=obj.row_id
        if row_id not in VarIndexRefView:
            VarIndexRefView[row_id]=[]
        
        VarIndexRefView[row_id].append(obj)

def get_var_ref_view(var_ref_seq):
    func_var_ref_view={}
    for obj in var_ref_seq:
        var_id=obj.var_global_id
        if var_id not in func_var_ref_view:
            func_var_ref_view[var_id]=[]
        
        func_var_ref_view[var_id].append(obj)
    
    return func_var_ref_view

def get_var_assign_view(var_store_seq):
    func_var_assign_view={}
    for var_store_obj in var_store_seq:
        var_id=var_store_obj.var_global_id
        depend_var_id=var_store_obj.depend_var_global_id
        
        if var_id !="" or var_id is not None:
            if var_id not in func_var_assign_view:
                func_var_assign_view[var_id]=[]
            func_var_assign_view[var_id].append(var_store_obj)    
        
        if depend_var_id !="" or depend_var_id is not None:
            pass
            #if depend_var_id not in VarPeekView:
            #    VarPeekView[depend_var_id]=[]
                
            #VarPeekView[depend_var_id].append(var_store_obj)
    return func_var_assign_view

def get_var_history_view(vars,var_assign_view,var_ref_view):
    def list_depend_var_by_location(var_id):
        if var_id not in VarAssignView:
            return {}
        
        variable_store_form=var_assign_view[var_id]
        variable_store_form=sorted(variable_store_form,key=lambda s:s.location,reverse=False)
        
        result=OrderedDict()
        for e in variable_store_form:
            if e.location not in result:
                result[e.location]=[]
            result[e.location].append(e)
        
        return result
    
    def list_var_ref_by_location(var_id):
        if var_id not in var_ref_view:
            return {}
        
        variable_ref_forms=var_ref_view[var_id]
        variable_ref_forms=sorted(variable_ref_forms,key=lambda s:s.location,reverse=False)
        
        
        result=OrderedDict()
        for e in variable_ref_forms:
            if e.location not in result:
                result[e.location]=[]
            result[e.location].append(e)
            
        return result
    var_assign_history={}
    var_ref_history={}
    for var in vars:
        var_id=var.global_id
        
        stmt_ref_view=list_var_ref_by_location(var_id)
        stmt_depend_var_view=list_depend_var_by_location(var_id)
        
        var_assign_history[var_id]=stmt_depend_var_view
        var_ref_history[var_id]=stmt_ref_view

    return var_assign_view,var_ref_view

def get_var(global_id):
    if global_id in VarTable:
        return VarTable[global_id]
    elif global_id in FuncTable:
        return FuncTable[global_id]
    else:
        return None

# def get_var_index(absolute_path,end_row,end_column):
#     index_of_file=VarIndexRefView.get(absolute_path,{})
    
#     return index_of_file.get((end_row,end_column))

def get_var_link(global_id):
    if global_id not in VarAssignHistoryView:
        return []
    link_var=set()
    stmt_view=VarAssignHistoryView[global_id]
    for line_number in stmt_view:
        vars=stmt_view[line_number]
        for var in vars:
            var_obj=get_var(var.depend_var_global_id)
            if var_obj not in link_var:
                link_var.add(var_obj)
                
    return list(link_var)

def pick_remarkable_key(table):
    number=20
    obj=None
    for key in table:
        if len(table[key])> number:
            number=len(table[key])
            obj=key
            break
    return obj

def pick_remarkable_function(table,text=""):
    if text=="":
        text=config.test_find_func
    select_range=list(table.keys())
    keys=[]
    for key in select_range:
        func_obj=table[key]
        if func_obj.name.find(text)!=-1:
            keys.append(key)
    return table[keys[1%len(keys)]]



def get_var_in_func(func_obj:da.FunctionDef):
    
    func_params=[]
    func_vars=[]
    
    if func_obj.get_function_owner_id() in FunctionOwnVarView:
        for var in FunctionOwnVarView[func_obj.get_function_owner_id()]:
            func_vars.append(var)
                
    if func_obj.get_function_owner_id() in FuncArgsTable:
        for func_param in FuncArgsTable[func_obj.get_function_owner_id()]:
            func_params.append(func_param)
    
    return func_params,func_vars

def get_func_abstract(func_obj:da.FunctionDef):
    func_params,func_vars=get_var_in_func(func_obj)
    owner_id=func_obj.get_function_owner_id()
    abstracts={}
    
    
    func_own_stores=[]
    func_own_refs=[]
    if owner_id in FunctionOwnVarStore:
        func_own_stores=FunctionOwnVarStore[owner_id]
    if owner_id in FunctionOwnVarRef:
        func_own_refs=FunctionOwnVarRef[owner_id]
    
    form_seq=func_own_refs+func_own_stores
    
    for form in form_seq:
        row_id=form.location
        
        if row_id not in abstracts:
            abstracts[row_id]=[]
        abstracts[row_id].append(form)
        
        var_indexs=VarIndexRefView.get(form.row_id)
        if var_indexs:
            abstracts[row_id]+=var_indexs
    return func_params,func_vars,abstracts

def view_func_abstract(func_obj:da.FunctionDef):
    
    tokens,cfg,counter=tokens_and_cfg(func_obj)
    line_identify="#NEWLINE#"
    unuse_word=['#start#','#end#']
    
    buffer=""
    for e in tokens:
        if e==line_identify:
            print(buffer)
            buffer=""
        elif e in unuse_word:
            pass
        else:
            buffer+=e
    
    if buffer!="":
        print(buffer)    

    for edge in cfg:
        print(tokens[edge[0]],tokens[edge[1]],edge[3])

    for node in counter:
        print(f"{node}:{counter[node]}")
        
def tokeize(func_obj:da.FunctionDef,include_func_vars=True,include_action=True):
    func_params,func_vars,row_abstracts=get_func_abstract(func_obj)
    tokens,_,counter=_tokeize(func_obj,func_params,func_vars,row_abstracts,include_func_vars,include_action)
    return tokens,counter

def _tokeize(func_obj,func_params,func_vars,row_abstracts,include_func_vars=True,include_action=True):
    def add_spliter(previous_tokens_length,tokens,splitor=const.line_spilter):
        if previous_tokens_length==len(tokens):
            tokens.append(splitor)
        else:
            tokens[-1]=splitor
        return len(tokens)
    
    var_indexes=[]
    tokens=[f'{const.bof}']
    var_counter=Counter()
    
    tokens+=[f'{func_obj.function_name}',f'{const.line_spilter}']
    now_length=len(tokens)
    for e in func_params:
            tokens+=[f'{e.name}',',']
            var_counter.update([e.name])
            var_indexes.append(len(tokens)-2)
            
            
    now_length=add_spliter(now_length,tokens)
    
    if include_func_vars:     
        for e in func_vars:
            tokens+=[f'{e.name}',',']
            var_indexes.append(len(tokens)-2)
        now_length=add_spliter(now_length,tokens)
              
    read_var_set=set()
    write_var_set=set()
    for row_id in sorted(list(row_abstracts.keys()),reverse=False):
        
        
        action_set=set()
        
        store_count=0
        ref_count=0
        
        for e in row_abstracts[row_id]:
            token_id=e.token_id
            if token_id not in read_var_set:
                text=""
                if isinstance(e,VariableStoreForm):
                    text=e.depend_var_text
                    store_count+=1
                    if e.is_substmt():
                        tokens+=[f"{e.var_text}",","]
                        var_counter.update([e.var_text])
                elif isinstance(e,VariableRefForm):
                    text=e.var_text
                    ref_count+=1
                elif isinstance(e,VariableIndex):
                    text=e.addon_text
                    ref_count+=1    
                if text != "":                    
                    tokens+=[f"{text}",","]
                    var_counter.update([text])
                    read_var_set.add(token_id)     
        
        if include_action:    
            tokens[-1]='|'
            
            
            for e in row_abstracts[row_id]:
                if e.expression_type != "":
                    if e.expression_type not in action_set:
                        tokens+=[f'{e.expression_type}',',']
                        action_set.add(e.expression_type)
            
                
        
        if store_count>0:
            tokens[-1]='>'           
            for e in row_abstracts[row_id]:
                if isinstance(e,VariableStoreForm):
                    token_id=e.var_token_id
                    if token_id not in write_var_set:                    
                        if e.is_substmt():
                            continue
                        if token_id not in write_var_set:
                            write_var_set.add(token_id)
                            var_counter.update([e.var_text])
                            tokens+=[f'{e.var_text}',',']
                    else:
                        pass
        tokens[-1]=const.line_spilter
        
    now_length=add_spliter(now_length,tokens,const.eof)
    
    return tokens,var_indexes,var_counter

def _cfg(func_obj,func_params,func_vars,row_abstracts):
    def next_syntax_edge(start_index,end_index,tokens,exclude_start=[],exclude_end=[]):
        edge_name="enum_NEXT_SYNTAX"
        if start_index is None or end_index is None:
            return None,end_index
        
        if tokens[start_index] in exclude_start:
            return None,end_index
        
        if tokens[end_index] in exclude_end:
            return None,start_index
        
        return [start_index,end_index,const.EDGE_TYPES[edge_name],edge_name],end_index
    
    def var_ref_edge(start_index,end_index):
        edge_name="enum_LAST_READ"
        
        return [start_index,end_index,const.EDGE_TYPES[edge_name],edge_name]
        
    def var_write_edge(start_index,end_index):
        edge_name="enum_LAST_WRITE"
        
        return [start_index,end_index,const.EDGE_TYPES[edge_name],edge_name]
    
    tokens,var_indexes,var_counter=_tokeize(func_obj,func_params,func_vars,row_abstracts,True,False)
    edges=[]
    
    #generate next syntax edge for cfg
    previous_index=None
    exclude_start=[const.eof,',',const.line_spilter]
    exclude_end=[const.bof,',']
    for i,e in enumerate(tokens):
        edge,previous_index=next_syntax_edge(previous_index,i,tokens,exclude_start,exclude_end)
        edges.append(edge) if edge else None    
    
    #generate var ref edge for cfg
    all_start_indexs=[e for e in var_indexes]
    all_start_var_name=[tokens[e] for e in var_indexes]
    for i,e in enumerate(tokens):
        #remove edge where start node same as end node
        if i in all_start_indexs:
            continue
        
        #retain edge with legal variable name
        start_index=all_start_var_name.index(e) if e in all_start_var_name else None
        if start_index is None:
            continue
        
        start_index=all_start_indexs[start_index]
        edge=var_ref_edge(start_index,i)
        edges.append(edge) if edge else None
    
    #generate var write edge for cfg
    in_var_write_line=False
    write_line_identification='>'
    line_end_identification=[const.eof,const.bof,const.line_spilter]
    exclude_word=[',']
    depend_var_indexes=[]
    
    for i,e in enumerate(tokens):
        if tokens[i] in line_end_identification:
            depend_var_indexes=[]
            in_var_write_line=False
        elif tokens[i] == write_line_identification:
            in_var_write_line=True
        elif tokens[i] in exclude_word:
            continue
        elif in_var_write_line:
            edges+=[var_write_edge(e,i) for e in depend_var_indexes]
        else:
            depend_var_indexes.append(i)    
            
    return edges,tokens,var_counter  

def tokens_and_cfg(func_obj:da.FunctionDef):
    func_params,func_vars,row_abstracts=get_func_abstract(func_obj)
    edges,tokens,node_counter=_cfg(func_obj,func_params,func_vars,row_abstracts)

    return tokens,edges,node_counter

def read_codeql_result(dir,include_source=True):
    function_file='{}/func.csv'.format(dir)
    variable_file='{}/var.csv'.format(dir)
    function_args_file='{}/func-args.csv'.format(dir)
    add_variable_file(variable_file)
    add_function_file(function_file)
    add_function_args_file(function_args_file)
    
    if include_source:
        variable_store_file='{}/var-write.csv'.format(dir)
        variable_ref_file='{}/var-read.csv'.format(dir)
        
        global VarStoreTable,VarRefTable
        
        add_variable_store_file(variable_store_file)
        add_variable_ref_file(variable_ref_file) 
    
    
def read_codeql_addon(dir):
    
    global VarIndexAddonTable
    
    variable_index_file='{}/var-subscript.csv'.format(dir)
    
    add_variable_index_file(variable_index_file)
    
def save_caches():
    data={}
    data["table"]={}
    data["table"]["VarTable"]=VarTable
    data["table"]["FuncTable"]=FuncTable
    data["table"]["FuncArgsTable"]=FuncArgsTable
    data["table"]["VarStoreTable"]=VarStoreTable
    data["table"]["VarRefTable"]=VarRefTable
    data["table"]["AbsolutePathMapping"]=da.AbsolutePathMapping
    data["table"]["AbsolutePathMappingReverse"]=da.AbsolutePathMappingReverse
    data["view"]={}
    
    data["view"]["FuncViewByOwnerId"]=FuncViewByOwnerId
    data["view"]["VarAssignView"]=VarAssignView
    data["view"]["VarPeekView"]=VarPeekView
    data["view"]["VarAssignHistoryView"]=VarAssignHistoryView
    data["view"]["FunctionOwnVarView"]=FunctionOwnVarView
    data["view"]["VarRefHistoryView"]=VarRefHistoryView
    data["view"]["VarRefView"]=VarRefView
    data["view"]["FunctionOwnVarStore"]=FunctionOwnVarStore
    data["view"]["FunctionOwnVarRef"]=FunctionOwnVarRef
    
    data["addon"]={}
    data["addon"]["VarIndexAddonTable"]=VarIndexAddonTable
    data["addon"]["VarIndexRefView"]=VarIndexRefView
    f=open(config.caches_path,'wb')
    pickle.dump(data,f)
    f.close()

def load_caches():
    global VarTable,VarStoreTable,FuncArgsTable,FuncTable,VarRefTable
    
    if os.path.exists(config.caches_path):
        f=open(config.caches_path,'rb')
        data=pickle.load(f,fix_imports=True)
        f.close()
        
        td=data["table"]
        
        
        VarTable=td["VarTable"]
        VarStoreTable=td["VarStoreTable"]
        FuncArgsTable=td["FuncArgsTable"]
        FuncTable=td["FuncTable"]
        VarRefTable=td["VarRefTable"]
        da.AbsolutePathMapping=td["AbsolutePathMapping"]
        da.AbsolutePathMappingReverse=td["AbsolutePathMappingReverse"]
        
        if config.caches_include_viewer:
            global VarPeekView,VarAssignView,VarAssignHistoryView,FunctionOwnVarView
            global FuncViewByOwnerId,VarRefView,VarRefHistoryView
            global FunctionOwnVarRef,FunctionOwnVarStore
            vd=data["view"]
            VarPeekView=vd["VarPeekView"]
            VarAssignView=vd["VarAssignView"]
            VarAssignHistoryView=vd["VarAssignHistoryView"]
            FunctionOwnVarView=vd["FunctionOwnVarView"]
            FuncViewByOwnerId=vd["FuncViewByOwnerId"]
            VarRefHistoryView=vd["VarRefHistoryView"]
            VarRefView=vd["VarRefView"]
            FunctionOwnVarStore=vd["FunctionOwnVarStore"]
            FunctionOwnVarRef=vd["FunctionOwnVarRef"]
        
        if config.caches_include_addon:
            addon=data.get("addon",{})
            global VarIndexAddonTable,VarIndexRefView
            VarIndexAddonTable=addon.get("VarIndexAddonTable",set())
            VarIndexRefView=addon.get("VarIndexRefView",{})
        return True
    
    return False 

def build_table_caches(dir):
    read_codeql_result(dir)
    data={}
    data["table"]={}
    data["table"]["VarTable"]=VarTable
    data["table"]["FuncTable"]=FuncTable
    data["table"]["FuncArgsTable"]=FuncArgsTable
    data["table"]["VarStoreTable"]=VarStoreTable
    data["table"]["VarRefTable"]=VarRefTable
    data["view"]={}
        
        
    f=open(config.caches_path,'wb')
    pickle.dump(data,f)
    f.close()

def init_for_test(db_dir):
    process_start_time=time.time()
    read_codeql_result(db_dir,False)
    build_variable_viewer()
    finish_time=time.time()
    print("init variable finish. time cost: {}".format(finish_time-process_start_time))

def clear_memory():
    if config.clear_memory:
        global VarRefTable,VarStoreTable,VarIndexAddonTable
        del VarRefTable
        del VarStoreTable
        del VarIndexAddonTable     

def init(db_dir):
    process_start_time=time.time()
    if config.rebuild_caches or not load_caches():
        read_codeql_result(db_dir)
        read_codeql_addon(db_dir)
        build_variable_viewer()
        
        save_caches()
        clear_memory()
    else:
        if not config.caches_include_addon:
            read_codeql_addon(db_dir)
        
        if not config.caches_include_viewer:
            build_variable_viewer()

        clear_memory()
    finish_time=time.time()
    print("init variable finish. time cost: {}".format(finish_time-process_start_time))
            
if __name__=="__main__":
    
    
    process_start_time=time.time()
    
    init(config.query_dir)
        
    build_index_finish_time=time.time()
    print("build index time: {:3f}".format(build_index_finish_time-process_start_time))
    
    func_obj=pick_remarkable_function(FuncTable)
    print(func_obj.get_function_owner_id())
    
    view_func_abstract(func_obj)
    
    
    finish_time=time.time()
    print("process total time cost: {}".format(finish_time-process_start_time))
    # target_var_id=pick_remarkable_key(VarStoreTable)
    # target_var=get_var(target_var_id)
    # print("path: {}".format(target_var.absolute_path))
    # print("target-var: {}".format(target_var_id))
    # link_vars=get_var_link(target_var_id)
    # for e in link_vars:
    #     print(e.global_id)
        
    # stmt_view=VarFormStmtTable[target_var_id]
    # for line_number in stmt_view:
    #     print("Rows: {}".format(line_number))
    #     for e in stmt_view[line_number]:
    #         print(e.depend_var_global_id)
    # for key in VarStoreTable:
    #     print(key,VarStoreTable[key])